{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入数据集\n",
    "# 数据集 -> 右边图标点击 -> 插入代码\n",
    "\n",
    "\n",
    "# 全量运行之后，获取训练完成的模型文件或者是中间文件的主要流程：\n",
    "# 上传模型文件 -> 申报审核文件 -> 审核通过 ->下载文件\n",
    "\n",
    "# 上传模型文件接口示例：\n",
    "# import wfio\n",
    "# server_path是上传到服务器的文件名称，可以自定义；local_path是本地上传的文件路径, 必须是本地已经存在的文件路径。\n",
    "# wfio.upload_to_oss(server_path,local_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd \n",
    "import numpy as np \n",
    "\n",
    "CHUNK_SIZE = 1000\n",
    "TRAIN_PARTIAL = 0.3\n",
    "EPOCH = 5\n",
    "\n",
    "df = pd.read_csv(\"ten_million_top1k.csv\") # 这行在运行环境时候应该注释掉\n",
    "\n",
    "# 然后得计算一些变量\n",
    "length = len(df)\n",
    "iter_size = int(length / CHUNK_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "\n",
    "def get_metric(y_test, y_pred):\n",
    "    # 设计成 dict 比较好加新的 metric，要不然还得改返回值\n",
    "    metrics_dict = {} \n",
    "    metrics_dict['accuracy'] = metrics.accuracy_score(y_test, y_pred)\n",
    "    metrics_dict['quadratic_weighted_kappa'] = metrics.cohen_kappa_score(y_test, y_pred, weights= 'quadratic')\n",
    "    return metrics_dict\n",
    "\n",
    "def display_metrics(metrics_dict):\n",
    "    metric_list = list(metrics_dict.keys())\n",
    "    data_list  = []\n",
    "    for m in metric_list:\n",
    "        data_list.append([m, metrics_dict[m]])\n",
    "    return pd.DataFrame(data_list, columns= ['Evaluation Metric', \"Value\"])\n",
    "\n",
    "def get_partial_data(input_X, ratio = 0.7):\n",
    "    length = len(input_X)\n",
    "    return input_X[:int(length * ratio)], input_X[int(length * ratio):]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start Epoch: 0\n",
      "Start Epoch: 1\n",
      "Start Epoch: 2\n",
      "Start Epoch: 3\n",
      "Start Epoch: 4\n",
      "Training -> Done.\n"
     ]
    }
   ],
   "source": [
    "# 然后开始训练模型\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "import copy \n",
    "\n",
    "model = SGDClassifier()\n",
    "\n",
    "for i in range(iter_size):\n",
    "    st_index = i * CHUNK_SIZE\n",
    "    ed_index = st_index + CHUNK_SIZE\n",
    "    df_tmp   = df[st_index:ed_index]\n",
    "\n",
    "for i in range(EPOCH):\n",
    "    print('Start Epoch: ' + str(i))\n",
    "    for i in range(iter_size):\n",
    "        st_index = i * CHUNK_SIZE\n",
    "        ed_index = st_index + CHUNK_SIZE\n",
    "        df_tmp   = copy.copy(df[st_index:ed_index])\n",
    "        # 取数据\n",
    "        y = df_tmp['y']\n",
    "        df_tmp.drop([\"y\"], axis = 1, inplace = True)\n",
    "        X = df_tmp.to_numpy()\n",
    "        X_train = get_partial_data(X)[0]\n",
    "        y_train = get_partial_data(y)[0]\n",
    "        # 做增量学习\n",
    "        model.partial_fit(X,y, classes = np.unique(y))\n",
    "\n",
    "print('Training -> Done.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 训练完了就是预测过程\n",
    "\n",
    "##  这个不怎么占内存\n",
    "y_test = []\n",
    "y_pred = []\n",
    "\n",
    "## 然后具体进行预测\n",
    "#df_iterator = pd.read_csv('one_million.csv', chunksize= CHUNK_SIZE)\n",
    "for i in range(iter_size):\n",
    "    st_index = i * CHUNK_SIZE\n",
    "    ed_index = st_index + CHUNK_SIZE\n",
    "    df_tmp   = copy.copy(df[st_index:ed_index])\n",
    "    # 取数据\n",
    "    y = df_tmp['y']\n",
    "    df_tmp.drop([\"y\"], axis = 1, inplace = True)\n",
    "    X = df_tmp.to_numpy()\n",
    "    X_test_part = get_partial_data(X)[0]\n",
    "    y_test_part = get_partial_data(y)[0]\n",
    "    y_pred_part = model.predict(X_test_part)\n",
    "    # 保存结果\n",
    "    y_test.extend(y_test_part)\n",
    "    y_pred.extend(y_pred_part)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'              precision    recall  f1-score   support\\n\\n           0       0.97      0.91      0.94       350\\n           1       0.92      0.97      0.94       350\\n\\n    accuracy                           0.94       700\\n   macro avg       0.94      0.94      0.94       700\\nweighted avg       0.94      0.94      0.94       700\\n'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics import classification_report\n",
    "classification_report(y_true=y_test, y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Done!\n"
     ]
    }
   ],
   "source": [
    "print('Done!')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
